#!/usr/bin/env python
#-*- coding: utf-8 -*-
#@Time : 2017/11/22 15:09
#@Author : ChenMei

import requests
from bs4 import BeautifulSoup
import threading
import sys
import datetime
import MySQLdb

class nSpider(object):

    def __init__(self,urls,clasify,):
        self.urls = urls
        self.clasify = clasify
        print "init end"

    def run(self):
        print "thread starting"
        thre = threading.Thread(target=self.main, args=(self.urls,self.clasify))
        thre.start()

    '''
    从列表中获取新闻的url及该新闻产生的时间，比对新闻的时间和当前系统时间
    只保留今天产生的新闻并调用内容解析方法，然后调用存储文件方法将获取的文
    本内容存入文件中文件名以新闻题目+url的形式命名，中间以@符号作为连接符
    参数：urltimes是一个二维数组，内数组存储的是其url及产生时间
    '''
    def main(self,urltimes,clasify):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6',
            'Cookie': 'tt_webid=6487802632568473101',
        }
        allnews = []
        for urltime in urltimes:
            t = urltime["time"]
            print urltime["url"] + " " + t + "的新闻 "
            today = datetime.datetime.now().strftime('%m/%d/%Y')
            if t != today:
                continue
            url = urltime["url"]
            title,content = self.fetchContent(headers,url,clasify)
            if content is None or len(content) == 0:
                print "无法获取 " + url + " 的新闻内容，它的所属类别是 " + clasify
                continue
            path = "163test/" + clasify + "/" + title + ".txt"
            self.saveFile(path,content)

            """构建新闻数据结构，并存入数据库中"""
            news = {}
            news['title'] = title
            news['content'] = content
            news['url'] = url
            news['date'] = datetime.datetime.now().strftime('%Y-%m-%d')
            news['category'] = clasify
            news['commentNum'] = (int)(urltime["tienum"])
            allnews.append(news)
        self.sqlHandle(allnews)


    '''将新闻数据存入数据库中'''
    def sqlHandle(self,allnews):
        db = MySQLdb.connect(host='localhost', user="root", passwd="123456", db="opinionSupervision", charset="utf8")
        cursor = db.cursor()
        for news in allnews:
            print news["date"],news["title"],news["category"],news["commentNum"],news["url"]
            sql = """insert into news(date,title,content,category,commentsNum,url)
                    values('%s','%s','%s','%s',%d,'%s')""" %(news["date"],news["title"],news["content"],news["category"],news["commentNum"],news["url"])
            try:
                cursor.execute(sql)
                db.commit()
                print "写入成功"
            except:
                db.rollback()
                print "写入失败"
        db.close()

    '''
    根据不同的新闻类别，调用相对应的解析文本内容的方法
    并将文本内容存入相应的文件中，文件名为新闻题目
    para：headers,url,clasify
    return：title，content
    '''
    def fetchContent(self,headers, url, clasify):
        req = requests.get(url,headers)
        soup = BeautifulSoup(req.text,"html.parser")
        title = ""
        if soup.title != None:
            title = soup.title.string.replace(":",",").replace("/",",").replace("?",",").replace(" ","").replace('"','‘')
        content = None
        if clasify == "要闻":
            content = self.yaowenFetch(url,soup)
        elif clasify == "体育":
            content = self.sportsFetch(url,soup)
        elif clasify == "独家":
            content = self.dujiaFetch(url,soup)
        elif clasify == "社会":
            content = self.shehuiFetch(url,soup)
        elif clasify == "财经":
            content = self.moneyFetch(url,soup)
        elif clasify == "汽车":
            content = self.autoFetch(url,soup)
        elif clasify == "科技":
            content = self.techFetch(url,soup)
        elif clasify == "国内":
            content = self.guoneiFetch(url,soup)
        elif clasify == "娱乐":
            content = self.entFetch(url,soup)
        elif clasify == "时尚":
            content = self.ladyFetch(url,soup)
        elif clasify == "健康":
            content = self.healthFetch(url,soup)
        elif clasify == "军事":
            content = self.warFetch(url,soup)
        elif clasify == "国际":
            content = self.internationalFetch(url,soup)
        contentStr = ""
        if content is not None:
            contentStr = content.text.strip()
        print title
        return title, contentStr

    '''
    将文本内容存入指定路径的文件中
    '''
    def saveFile(self,filePath,content):
        try:
            write = open(filePath,'w')
            write.write(content.strip())
        except:
            print "write " + filePath + " error"
    '''
    定义获取不同类别的新闻的方法，主要包括12大类
    要闻、体育、独家、社会、财经、汽车、军事、科技、国内、娱乐、时尚、健康
    接收参数为url及soup结构树
    return 包含文本内容的子结构树
    '''

    #获取国际新闻
    def internationalFetch(self,url,soup):
        if "news.163.com" in url:
            content = self.newsFetch(soup)
        else:
            content = None
        return content

    #获取要闻
    def yaowenFetch(self,url,soup):
        if "news.163.com" in url:
            content = self.newsFetch(soup)
        elif "dy.163.com" in url:
            content = self.dyFetch(soup)
        else:
            content = None
        return content

    #获取体育信息
    def sportsFetch(self,url,soup):
        if "nba.sports.163.com" in url:
            content = self.nbaFetch(soup)
        elif "sports.163.com" in url:
            content = self.newsFetch(soup)
        elif "hongcai.163.com" in url:
            content = self.hongcaiFetch(soup)
        else:
            content = None
        return content

    #获取独家新闻
    def dujiaFetch(self,url,soup):
        if "renjian.163.com" in url:
            content = self.renjianFetch(soup)
        elif "data.163.com" in url:
            content = self.dataFetch(soup)
        elif "caozhi.news.163.com" in url:
            content = self.caozhiFetch(soup)
        elif "view.163.com" in url:
            content = self.viewFetch(soup)
        else:
            content = None
        return content

    #获取社会新闻
    def shehuiFetch(self,url,soup):
        content = self.newsFetch(soup)
        return content

    #获取财经新闻
    def moneyFetch(self,url,soup):
        if "money.163.com" in url:
            content = self.newsFetch(soup)
        else:
            content = None
        return content

    #获取汽车新闻
    def autoFetch(self,url,soup):
        if "auto.163.com" in url:
            content = self.newsFetch(soup)
        elif "dy.163.com" in url:
            content = self.dyFetch(soup)
        else:
            content = None
        return content

    #获取军事新闻
    def warFetch(self,url,soup):
        if "war.163.com" in url:
            content = self.newsFetch(soup)
        elif "dy.163.com" in url:
            content = self.dyFetch(soup)
        else:
            content = None
        return content

    #获取科技新闻
    def techFetch(self,url,soup):
        if "tech.163.com" in url:
            content = self.newsFetch(soup)
        elif "dy.163.com" in url:
            content = self.dyFetch(soup)
        else:
            content = None
        return content

    #获取国内新闻
    def guoneiFetch(self,url,soup):
        if "news.163.com" in url:
            content = self.newsFetch(soup)
        else:
            content = None
        return content

    #获取娱乐新闻
    def entFetch(self,url,soup):
        if "ent.163.com" in url:
            content = self.newsFetch(soup)
        else:
            content = None
        return content

    #获取时尚新闻
    def ladyFetch(self,url,soup):
        content = self.newsFetch(soup)
        return content

    #获取健康新闻
    def healthFetch(self,url,soup):
        if "jiankang.163.com" in url or "news.163.com" in url:
            content = self.newsFetch(soup)
        else:
            content = None
        return content

    '''
    因为有些不同主题的新闻页面结构相同，而又有些不同主题的新闻页面结构完全不同，
    因此定义了几种不同类型页面提取文本的解析方法，需严格判断页面的类型以调用正确的方法
    接受参数为soup结构树
    return  包含文本内容的子结构树
    '''
    def newsFetch(self,soup):
        content = soup.find('div', class_='post_text')
        return content

    def dyFetch(self,soup):
        content = soup.find('div',class_='content')
        return content

    def nbaFetch(self,soup):
        content = soup.find('div',class_='endText')
        return content

    def hongcaiFetch(self,soup):
        content = soup.find('div',class_='content')
        return content

    def renjianFetch(self,soup):
        content = soup.find('div',class_='endText')
        content.script.extract()
        return content

    def dataFetch(self,soup):
        content = soup.find('div',class_='main-content')
        return content

    def caozhiFetch(self,soup):
        content = soup.find('div',class_='endText')
        return content

    def viewFetch(self,soup):
        content = soup.find('div',class_='resound_main_content')
        return content
